<?php
//-----------------------------------------------------------------------------------------
//
//	Copyright (c) 2011 Iguana-Farm.com LLC
//
//-----------------------------------------------------------------------------------------
error_reporting(E_ALL);

require_once("config.php");

define('DEBUG_LEVEL', 	1);
define('SHMEM_KEY', 	0x1235);
define("SSH", 			"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q ");
define("LOCK_FILE", 	"monitor.lockfile");

putenv('EC2_PRIVATE_KEY=' . AWS_KEY_FILE);
putenv('EC2_CERT=' . AWS_CERT_FILE);

//
//	Make sure this only runnining only once
//
$fp = fopen(LOCK_FILE, "w+");
if (!$fp)
    log_error("unable to create lock file");
	
if (!flock($fp, LOCK_EX | LOCK_NB)) 
    log_error("already running");

//
//	Initialize some stuff
//
$servers = array(MASTER, SLAVE);
$cmon = array();

//
//	Get task to monitor
//
if (!isset($_SERVER["argv"][1]))
	log_error("Usage: " . $_SERVER["PHP_SELF"] . " <check|check-and-failover|make_slave>\n");

$task = $_SERVER["argv"][1];

//
//	Create the shared memory so we can remember the number of consecutive errors
//
$shmem_id = shm_attach(SHMEM_KEY);
if ($shmem_id === false) 
    log_error('Unable to create the shared memory segment');

if (!shm_has_var($shmem_id, 1))
	{
	$master_errors = 0;
	shm_put_var($shmem_id, 1, $master_errors);

	$slave_errors = 0;
	shm_put_var($shmem_id, 2, $slave_errors);
	}

$master_errors = shm_get_var($shmem_id, 1);
$slave_errors  = shm_get_var($shmem_id, 2);

//
//	Check if all servers are running
//
foreach ($servers as $server)
	{
	$cmon[$server]["ip"] = gethostbyname($server);
	
	$cmon[$server]["ping"]  = false; 
	$cmon[$server]["mysql"] = false; 
		
	$rc = "";
	system("ping -q -c1 $server &> /dev/null", $rc);
	
	if ($rc != 0)
		{
		log_note("$server is not responding to ping");
		}
	else
		{
		$cmon[$server]["ping"]  = true;
		
		//
		//	Check if mysql is running
		//
		$rc = @mysql_connect($server , MYSQL_USER, MYSQL_PASSWD);
		if ($rc)
			{
			$cmon[$server]["mysql"]  = true;
			mysql_close($rc);
			}
		else
			{
			$cmon[$server]["mysql_errno"]  = mysql_errno();
			$cmon[$server]["mysql_error"]  = mysql_error();

			log_note("$server mysql error " . mysql_error() . " ("  . mysql_errno() . ")" );
			}
		}
	}

//
//	Evaluate errors and uptate the internal counters
//
if ($cmon[MASTER]["ping"] && $cmon[MASTER]["mysql"])		
	{
	$master_errors = 0;
	}

if ($cmon[SLAVE]["ping"] && $cmon[SLAVE]["mysql"])
	{
	$slave_errors = 0;
	}

$master_errors += (!$cmon[MASTER]["ping"] || !$cmon[MASTER]["mysql"]) ? 1 : 0;
$slave_errors  += (!$cmon[SLAVE]["ping"] || !$cmon[SLAVE]["mysql"] ) ? 1 : 0;

shm_put_var($shmem_id, 1, $master_errors);
shm_put_var($shmem_id, 2, $slave_errors);

log_note("Error count: Master $master_errors, Slave $slave_errors");

//
//	If the master is down and the slave is running, we need to consider a failover
//
switch ($task)
	{
//-----------------------------------------------------------------------------------------
//
//	Check cluster status and do failover if needed
//
//-----------------------------------------------------------------------------------------
case "check":
case "check-and-failover":

	//	Check replication is running
	if ($master_errors == 0 && $slave_errors == 0)
		{
		$rc = @mysql_connect(SLAVE , MYSQL_USER, MYSQL_PASSWD);
		if ($rc)
			{
			$result = mysql_query("SHOW SLAVE STATUS");
			if (!$result)
				log_error("Unable to get slave status");
			
			$row = mysql_fetch_assoc($result);
			if (!$row)
				log_error("Unable to get slave status");
				
			if ($row["Slave_IO_Running"] != "Yes")
				log_error("Slave_IO_Running not running");

			if ($row["Slave_SQL_Running"] != "Yes")
				log_error("Slave_SQL_Running not running");

			if ($row["Seconds_Behind_Master"] > 30)
				log_error("Slave running behind master: " . $row["Seconds_Behind_Master"]);

			mysql_close($rc);
			}
		}

	if ($master_errors || $slave_errors)
		{
		log_note("Someting is not well...");
		}

	//
	//	If the slave is down and the master is up, recreate the slave 
	//
	if ($task == "check-and-failover" &&
			!$cmon[SLAVE]["ping"] && $slave_errors >= 3  && 
			$cmon[MASTER]["ping"] && $cmon[MASTER]["mysql"])
		{

		}
		
	//
	//	Initiate a slave to master failover
	//
	if ($task == "check-and-failover" 
		&& !$cmon[MASTER]["ping"] && $master_errors >= 3  && 
			$cmon[SLAVE]["ping"] && $cmon[SLAVE]["mysql"])
		{
		log_note("Failover slave to master process started...");
		
		//
		//	On error, collect some more instance infromation 
		//
		cmon_ec2_info();
	
		//
		//	Get coordinates of all servers involved
		//
		if (!isset($cmon[SLAVE]["instance_id"] ))
			{
			log_error("Failover failed since we could not get slave information");
			}
		
		$old_master_id = "";
		if (isset($cmon[MASTER]["instance_id"]))
			$old_master_id = $cmon[MASTER]["instance_id"];

		$new_master_id = $cmon[SLAVE]["instance_id"];

		//
		//	Log into the slave mysql and stop replication
		//
		$rc = @mysql_connect(SLAVE , MYSQL_USER, MYSQL_PASSWD);
		if ($rc)
			{
			mysql_query("STOP SLAVE");
			mysql_query("RESET MASTER");
			mysql_query("RESET SLAVE");
	
			mysql_close($rc);
			}
		else
			{
			log_error("Unable to stop replication on slave to be promoted to master");
			}

		//	
		//	Do some other cleanups on the new master
		//		
		ssh_command(SLAVE, "cp /etc/my.cnf /etc/my.cnf.bak");
		ssh_command(SLAVE, "cat /etc/my.cnf.bak | sed 's/skip-slave/#skip-slave-start/' > /etc/my.cnf");
		ssh_command(SLAVE, "rm -rf " . SLAVE_TAG_NAME);
		ssh_command(SLAVE, "touch "  . MASTER_TAG_NAME);

		//
		//	Kill the old master if he is still around
		//
		kill_by_tag(MASTER_TAG_NAME);

		sleep(5);
		
		command("ec2-associate-address -i $new_master_id " . MASTER_ELASTIC_IP);
		command("ec2-create-tags $new_master_id --tag Name=" . MASTER_TAG_NAME);
		}

	break;	
	

//-----------------------------------------------------------------------------------------
//
//	Create slave from master 
//
//-----------------------------------------------------------------------------------------
case "make_slave":

	if (!$cmon[MASTER]["ping"] || !$cmon[MASTER]["mysql"])
		log_error("Master mysql must be reachable and running");

	//
	//	Collect other info
	//
	cmon_ec2_info();

	//
	//	Get the name of the actual AMI
	//
	$cmd = "ec2-describe-images | grep TAG | grep " . AMI_TAG;
	$image_info = preg_split("/\t/", command($cmd));

	if (!isset($image_info[2]))
		log_error("Unexpected output from ec2-describe-images");

	$ami_id = $image_info[2];

	//
	//	Kill the old slaves if he is still around
	//
	kill_by_tag(SLAVE_TAG_NAME);

	//
	//	Create a snapshot on the master
	//
	$output = ssh_command(MASTER, "/usr/bin/php /vol/bin/ebs-snapshot.php");

	if (!preg_match("/Backup: (.+) (.+) (.+)\n/", $output, $match))
		{
		log_error("Unexpected value from snapshot command\n$output");
		}
	
	$snap_id = $match[3];

	//
	//	Wait for the snapshot to finish and get the master log information
	//
	$snap_info = wait_for_ec2("ec2-describe-snapshots", $snap_id, "SNAPSHOT", "completed", 3);


	if (!preg_match("/- (.+) at ([0-9]+)$/", $snap_info[8], $match))
		log_error("Unable to get master log info from snapshot description " . $snap_info[8]);
		
	$master_log_file = $match[1];
	$master_log_pos = $match[2];
	
	//
	//	Launch an AMI on the zone where the master is not currently at
	//
	$new_zone = ($cmon[MASTER]["zone"] == ZONE_1) ? ZONE_2 : ZONE_1;
	
	$cmd = "ec2-run-instances ".
			" -n 1 -g " .SECURITY_GROUP. 
			" -k " . KEY_PAIR . 
			" -t " . INSTANCE_TYPE . 
			" --availability-zone $new_zone " . 
			$ami_id . " | grep INSTANCE";
	
	$instance_info = preg_split("/\t/", command($cmd));

	if (!isset($instance_info[1]))
		log_error("Unexpected output from run instance command");

	$instance_id = $instance_info[1];
	
	//
	//	Create a volume in the new zone with the snapshot
	//
	$cmd = "ec2-create-volume --snapshot $snap_id  --availability-zone $new_zone | grep VOLUME";
	$volume_info = preg_split("/\t/", command($cmd));

	if (!isset($volume_info[1]))
		log_error("Unexpected output from ec2-create-volume");

	$volume_id = $volume_info[1];

	//
	//	Wait for the instance to be in running state
	//
	$instance_info = wait_for_ec2("ec2-describe-instances", $instance_id, "INSTANCE", "running", 5);

	$instance_ip = $instance_info[3];
	
	//
	//	Wait for the volume to be in running state
	//
	$volume_info = wait_for_ec2("ec2-describe-volumes", $volume_id, "VOLUME", "available", 5);
	
	//
	//	Attach volume to instance
	//
	command("ec2-attach-volume $volume_id -i $instance_id -d /dev/sdf");
	
	//
	//	Wait for attachment to be ready
	//
	$volume_info = wait_for_ec2("ec2-describe-volumes", $volume_id, "VOLUME", "in-use", 5);
	
	sleep(5);
	
	//
	//		Now mount the volume and run the provision script
	//
	ssh_command($instance_ip, "mount /dev/sdf /vol");
	ssh_command($instance_ip, "/usr/bin/php /vol/bin/provision.php");
	
	//
	//	Now we should be able to mysql into the instance and start the slave process at the correct place
	//
	$rc = @mysql_connect($instance_ip , MYSQL_USER, MYSQL_PASSWD);
	if ($rc)
		{
		mysql_query("STOP SLAVE");
		mysql_query("RESET MASTER");
		mysql_query("RESET SLAVE");

		mysql_query("CHANGE MASTER TO MASTER_LOG_FILE='$master_log_file', MASTER_LOG_POS=$master_log_pos");

		mysql_query("START SLAVE");

		mysql_close($rc);
		}
	else
		{
		log_error("Unable start replication on slave");
		}

	//
	//		Comment out skip-slave-start so next time server start...
	//
	ssh_command($instance_ip, "cp /etc/my.cnf /etc/my.cnf.bak");
	ssh_command($instance_ip, "cat /etc/my.cnf.bak | sed 's/skip-slave/#skip-slave-start/' > /etc/my.cnf");

	//
	//	Finally we can change the elastic IP of the slave this instance
	//
	command("ec2-associate-address -i $instance_id " . SLAVE_ELASTIC_IP);

	command("ec2-create-tags $instance_id --tag Name=" . SLAVE_TAG_NAME);

	//
	//	Associated this instance with the ELB
	//
	command("elb-register-instances-with-lb " . ELB . " --instances $instance_id");

	break;
	
//-----------------------------------------------------------------------------------------
//
//	Nothing to be done
//
//-----------------------------------------------------------------------------------------
default:
	log_error("Do not know how to $task");
	break;
	}
	

die("Done\n");


//-----------------------------------------------------------------------------------------
//
//
//-----------------------------------------------------------------------------------------
function command($cmd)
	{
	if (DEBUG_LEVEL > 0)
		log_note($cmd);

	$rc = 0;
	$output = array();
	
	exec($cmd, $output, $rc);

	return join("\n", $output);
	}
	
//-----------------------------------------------------------------------------------------
//
//
//-----------------------------------------------------------------------------------------
function wait_for_ec2($ec2_cmd, $id, $grep, $exit_state, $state_idx = 3, $sleep_time = 10)
	{
	log_note("Doing $ec2_cmd on $id");
	
	$info = array();
	
	while (true)
		{
		$info  = preg_split("/\t/", command("$ec2_cmd $id | grep $grep"));
		if (!isset($info[3]))
			log_error("Unexpected output from $ec2_cmd");
		
		if ($info[$state_idx] == $exit_state)
			break;

		log_note("Waiting for  $ec2_cmd on $id to be $exit_state... " . $info[$state_idx] );
		sleep($sleep_time);
		}	

	return $info;
	}
	
//-----------------------------------------------------------------------------------------
//
//
//-----------------------------------------------------------------------------------------
function ssh_command($ip, $cmd)
	{
	$rc = 0;
	$output = array();
	
	$cmd = SSH . " $ip  '$cmd'";

	if (DEBUG_LEVEL > 0)
		log_note($cmd);
		
	exec($cmd, $output, $rc);

	return join("\n", $output);
	}

//-----------------------------------------------------------------------------------------
//
//
//-----------------------------------------------------------------------------------------
function log_note($str)
	{
	print($str  . "\n");
	}

//-----------------------------------------------------------------------------------------
//
//
//-----------------------------------------------------------------------------------------
function log_error($str)
	{
	log_note($str);
	exit(1);
	}

//-----------------------------------------------------------------------------------------
//
//
//-----------------------------------------------------------------------------------------
function cmon_ec2_info()
	{
	global $cmon;

	log_note("Collecting instance information");
	
	//
	//	On error, collect some more instance infromation 
	//
	$instances  = preg_split("/\n/", command("ec2-describe-instances | grep INSTANCE"));
	foreach ($instances as $line)
		{
		$fields = preg_split("/\t/", trim($line));
		if (!isset($fields[17]))
			continue;
		
		$id = $fields[1];
		
		$instance[$id]["id"] 			= $id;
		$instance[$id]["private_dns"] 	= $fields[4];
		$instance[$id]["status"]	 	= $fields[5];
		$instance[$id]["zone"] 			= $fields[11];
		$instance[$id]["public_ip"] 	= $fields[16];
		$instance[$id]["private_ip"] 	= $fields[17];
		
		foreach ($cmon as $server => $server_data)
			{
			if ($server_data["ip"] == $instance[$id]["private_ip"])
				{
				$cmon[$server]["instance_id"] = $id;
				$cmon[$server]["public_ip"]   = $instance[$id]["public_ip"];
				$cmon[$server]["zone"]   = $instance[$id]["zone"];
				}
			}
		}
	}
	
//-----------------------------------------------------------------------------------------
//
//
//-----------------------------------------------------------------------------------------
function kill_by_tag($tag)
	{	
	//
	//	Kill instancs that have the tag
	//
	$old_instances = command("ec2-describe-instances | grep TAG | grep " . $tag);

	if (preg_match_all("/instance\t([^\t]+)\t/", $old_instances, $match))
		{
		foreach ($match[1] as $old_id)
			{
			command("ec2-terminate-instances $old_id");
			command("ec2-create-tags $old_id --tag Name=monitor-kill");
			command("elb-deregister-instances-from-lb " . ELB . " --instances $old_id");
			}
		}
	}
	
?>
